package learning.data;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;

/*
 * Converts a dataset from Kylin's format into a format that is understood
 * by Capisco.
 */
public class KylinDatasetConverter {
	static String infoboxClass = "comedian";
	static String attribute = "nationality";
	
	static String input = "O:/unix/projects/pardosa/data04/wufei/TRWiki/experiments/infobox-completion/" + infoboxClass + "/20070716/TrainForCRF/" + attribute + "_Train_Ca_Neg_DC_Bi";
	static String outputFeatures = "O:/unix/projects/pardosa/data10/raphaelh/learning/" + infoboxClass + "_" + attribute + ".fts";
	static String outputDataset = "O:/unix/projects/pardosa/data10/raphaelh/learning/" + infoboxClass + "_" + attribute + ".data";
	static String outputLabels = "O:/unix/projects/pardosa/data10/raphaelh/learning/" + infoboxClass + "_" + attribute + ".labels";
	
	public static void main(String[] args) throws IOException {
		
		// part 1: create features and labels		
		{
			HashSet<String> features = new HashSet<String>();
			HashSet<String> labels = new HashSet<String>();

			BufferedReader br = new BufferedReader(new InputStreamReader
					(new FileInputStream(input), "utf-8"));
			String line = null;
			
			while ((line = br.readLine()) != null) {
				if (line.length() == 0) continue;
				
				String[] fvs = line.split(" ");
				
				// the last col is the label
				for (int i = 0; i < fvs.length - 1; i++) {
					String fv = fvs[i];
					if (!features.contains(fv)) features.add(fv);
				}
				if (!labels.contains(fvs[fvs.length-1])) labels.add(fvs[fvs.length-1]);
			}
			
			String[] fa = features.toArray(new String[0]);
			Arrays.sort(fa);
			
			{
				BufferedWriter bw = new BufferedWriter(new OutputStreamWriter
						(new FileOutputStream(outputFeatures), "utf-8"));
				int id = 0;
				for (String f : fa) {
					bw.write(id++ + "\t" + f + "\n");
				}
				bw.close();
			}
			
			{
				BufferedWriter bw = new BufferedWriter(new OutputStreamWriter
						(new FileOutputStream(outputLabels), "utf-8"));
				int id = 0;
				for (String l : labels) {
					bw.write(id++ + "\t" + l + "\n");
				}
				bw.close();
			}
		}
		
		// part 2: convert data
		{
			HashMap<String, Integer> features = new HashMap<String, Integer>();
			HashMap<String, Integer> labels = new HashMap<String, Integer>();
			{
				BufferedReader br = new BufferedReader(new InputStreamReader
						(new FileInputStream(outputFeatures), "utf-8"));
				String line = null;
				while ((line = br.readLine()) != null)
					features.put(line.substring(line.indexOf("\t")+1), 
							Integer.parseInt(line.substring(0, line.indexOf("\t"))));
				br.close();
			}
			{
				BufferedReader br = new BufferedReader(new InputStreamReader
						(new FileInputStream(outputLabels), "utf-8"));
				String line = null;
				while ((line = br.readLine()) != null)
					labels.put(line.substring(line.indexOf("\t")+1), 
							Integer.parseInt(line.substring(0, line.indexOf("\t"))));
				br.close();
			}
			
			BufferedWriter bw = new BufferedWriter(new OutputStreamWriter
					(new FileOutputStream(outputDataset), "utf-8"));
			
			BufferedReader br = new BufferedReader(new InputStreamReader
					(new FileInputStream(input), "utf-8"));
			String line = null;
			
			while ((line = br.readLine()) != null) {

				if (line.length() == 0) {
					bw.write("\n");
					continue;
				}

				String[] fs = line.split(" ");
				
				bw.write(fs[0] + " " + labels.get(fs[fs.length-1]));
				
				int[] fids = new int[fs.length-1];
				for (int i=0; i < fs.length - 1; i++)
					fids[i] = features.get(fs[i]);
				Arrays.sort(fids);
				
				for (int i=0; i < fids.length; i++)
					bw.write(" " + fids[i] + ":1");
				bw.write("\n");
			}
			br.close();
			bw.close();
		}
	}
}
